from HTMLParser import HTMLParser
import re_db
import logging
import urllib2
import re

#from google.appengine.api import urlfetch
#from lxml import html

#logging.basicConfig(level=logging.WARNING)
logging.basicConfig(level=logging.DEBUG)
#re_non_word_non_num="[\\:\\(\\[\\{\\s\\)\\]\\}\\+\\-]+"

'''
class MLStripper(HTMLParser):
    def __init__(self):
        self.reset()
        self.fed = []
    def handle_data(self, d):
        self.fed.append(d)
    def get_data(self):
        return ''.join(self.fed)

def strip_tags(html):
    s = MLStripper()
    s.feed(html)
    return s.get_data()

def htmlToPlainText(urlInput):
    try:
        result = urlfetch.fetch(urlInput)
        if result.status_code == 200:
            logging.info('htmlToPlainText: get url "%s" success' % urlInput)
            htmlDoc = html.fromstring(result.content)
            plainText = htmlDoc.text_content()
            return plainText
    except:
        logging.error('htmlToPlainText: get url "%s" failed!' % urlInput)
        pass
    return None
'''
def htmlToPlainText(urlInput):    
    #url_to+match="http://ca.finance.yahoo.com/q/mh?s=FTNT"
    #search_value="87%"
    logging.info("in htmlToPlainText,start to fetch url")
    req = urllib2.Request(url=urlInput)
    response = urllib2.urlopen(req)
    #html_resp = response.read().decode("utf-8")
    html_resp = response.read()
    logging.info("in htmlToPlainText,finish fetching url")
    logging.info("in htmlToPlainText,length of response is %s" % len(html_resp))
    
    #document = lxml.html.document_fromstring(str)
    #raw_text = document.text_content()
    #f1=open('/tmp/re_original.html','w+')
    #f2=open('/tmp/re_remove_tag.txt','w+')
    #f3=open('/tmp/re_remove_newline.txt','w+')
    #f4=open('/tmp/re_remove_parenthesis.txt','w+')
    #raw_str = strip_tags(str)

    raw_str_no_tag = re.sub('<[^<]+?>','   ',html_resp)
    #regular expression can only work for one line, so we remove \r\n to make whole page as one line
    #raw_str_no_newline=raw_str_no_tag.replace('\r',' SUBNEWLINE ').replace('\n',' SUBCARRIAGERETURN ')
    raw_str_no_newline = raw_str_no_tag.replace('\r',' ').replace('\n',' ').replace('&',' ')
    #f1.write(html_resp)
    #f2.write(raw_str_no_tag)
    #f3.write(raw_str_no_newline)
    
    #parenthesis may be choosed and inserted to created re_string, but it may break re integrity. so here we decide to remove it
    plainText = raw_str_no_newline.replace('(','').replace(')','').replace('[','').replace(']','').replace('{','').replace('}','')
    #f4.write(plainText)
    #f1.close()
    #f2.close()
    #f3.close()
    #f4.close()
    #logging.debug("---------------------------------------------")
    #logging.debug(raw_str)
    return plainText

def re_generator_by_str(str_to_match,match_value):
    re_group = {}
    match = {}
    ret_str = []
    i=1
    #re_non_word_non_num="[\\:\\(\\[\\{\\s\\)\\]\\}\\+\\-]+"
    if ("%" in match_value) and ("." in match_value):          matching_re="[\d\.]+%"
    elif ("M" in match_value) and ("." in match_value):        matching_re="[\d\.]+M"
    elif ("m" in match_value) and ("." in match_value):        matching_re="[\d\.]+m"
    elif ("%" in match_value) and ("," in match_value):        matching_re="[\d\,]+%"
    elif ("+" in match_value) and ("," in match_value):        matching_re="[\d\,]+\+"
    elif ("-" in match_value) and ("," in match_value):        matching_re="[\d\,]+\-"
    elif "%" in match_value:                                   matching_re="\d+%"
    elif "+" in match_value:                                   matching_re="\d+\+"
    elif "-" in match_value:                                   matching_re="\d+\-"
    elif "," in match_value:                                   matching_re="[\d\,]+"
    elif "." in match_value:                                   matching_re="[\d\.]+"
    elif "M" in match_value:                                   matching_re="\d+M"
    elif "m" in match_value:                                   matching_re="\d+m"
    else :                                                     matching_re="\d+"

    logging.debug("match_value is %s, matching_re is %s" % (match_value,matching_re))

    for key,value in re_db.re_int.items() :
        logging.debug("key is %s, value[0] is %s" % (key,value[0]))
        logging.debug("replacing matching_re")
        #re_value=value[0].replace("matching_value_re",matching_re)
        re_value=value[0].replace("matching_value_re",match_value)
        logging.debug("new value is %s" % re_value)
        re_group[1] = re.compile(re_value)
        logging.debug("finishing compile,start to re.finditer")
        for m in re.finditer(re_group[1], str_to_match) :
            #logging.debug m
            #logging.debug m.group(1)
            logging.debug("found one number. i is %s" % str(i))
            if m.group(2) == match_value:
                logging.debug("found one matching number. i is %s" % str(i))
                logging.debug("matching to Correct value: %s %s" % (m.group(1),m.group(2)))
                try:
                     ret_str_new = m.group(1)+re_db.re_non_word_non_num+"("+matching_re+")"+re_db.re_non_word_non_num+m.group(3)
                     ret_str.append(ret_str_new)
                except IndexError:
                     ret_str_new= m.group(1)+re_db.re_non_word_non_num+"("+matching_re+")"
                     ret_str.append(ret_str_new)
                i=i+1
            else:
                logging.debug("%s matching to wrong value: %s %s" % (key,m.group(1),m.group(2)))
                #logging.debug "group(1) is "+match[1].group(1)
                #logging.debug "group(2) is "+match[1].group(2)
                #logging.debug "group(3) is "+match[1].group(3)
                #ret_str=ret_str+"\D+\d+"
                #left_str=m.group(3)
                #logging.debug(left_str)
                #new_ret=re_generator(left_str,match_value)
                #ret_str=ret_str+new_ret
        else:
            logging.debug("no matching with re_group[1]: %s" % re_group[1])

    #recheck if re_string is ok to extract correct value
    ret_str_new=[]
    for re_string in ret_str:
        logging.info("re is %s" % re_string)
    
        re_obj = re.compile(re_string)
        match = re.search(re_obj,str_to_match)
        if match :
            if match.group(1) == match_value :
                logging.debug("GREAT!!!! %s" % match.group(1))
                ret_str_new.append(re_string)
            else:
                logging.warning("Fail to match, remove it! %s" % match.group(1))
        else :
            logging.error("From re_generator_by_string, BUG:This case should never happen.")
    return ret_str_new

def re_generator_by_url(urlInput, search_value):
    plainText = htmlToPlainText(urlInput)

    #html_new = re.sub('<[^<]+?>','',str)
    #logging.debug(html_new)    
    #str="aabb sddsdf 2342 dgsdd" 
    
    re_str_list = re_generator_by_str(plainText, search_value)    
    for re_string in re_str_list:
        logging.info("From re_gerator_by_url,re is "+re_string)    
        re_obj = re.compile(re_string)
        match = re.search(re_obj, plainText) 
        if match :
            if match.group(1) == search_value :
                logging.info("From re_gerator_by_url,GREAT!!!! %s" % match.group(1))
            else:
                logging.error("Bad! %s" % match.group(1))
        else :
            logging.error("From re_gerator_by_url, BUG:This case should never happen.")
    return re_str_list

def re_generator_apply(urlInput, regexp):
    plainText = htmlToPlainText(urlInput)
    reObj = re.compile(regexp)
    m = re.search(reObj, plainText)
    if m and m.group(1):
        return int(float(m.group(1).replace(',','').replace('%','').replace('+','').replace('-','')))
    return None
